'''
Created on Jan 7, 2010

@author: Roni
'''
import orchid
import logging
import sys
import webgraphs.webgraphCrawler
import webgraphs.wikiAnalyzer
import webgraphs.citeseerAnalyzer
import webgraphs.googleScholarAnalyzer
import cliques.utils
import cliques.cliqueFinder
import igraph
import os
import random
import time
import timerManager
           
RESOURCE_WEBGRAPH_DIR= "../resources/graphs/web"
    
def main():

    logging.basicConfig(level=logging.DEBUG)
    logging.root.addHandler(logging.StreamHandler(sys.stdout))
    
    seeds = {
             'GS-Heuristic-MultiAgent':['http://scholar.google.com/scholar?q=MultiAgent'],
             'GS-Heuristic-Israel':['http://scholar.google.com/scholar?q=Israel'],
             'GS-Arificial-Intelligence-Clique':['http://scholar.google.com/scholar?q=Arificial+Intelligence+Clique'],
             'GS-Heuristic-Online-Algorithms':['http://scholar.google.com/scholar?q=Online+Algorithms'],
             'GS-Heuristic-Robot-Navigation':['http://scholar.google.com/scholar?q=Robot+Navigation'],
             'GS-Subgraph-Isomorphism':['http://scholar.google.com/scholar?q=Subgraph+Isomorphism'],
             'GS-webgraph':['http://scholar.google.com/scholar?q=webgraph'],
             'GS-Social-Graph':['http://scholar.google.com/scholar?q=Social+Graph'],
             'GS-Natural-Language-Processing':['http://scholar.google.com/scholar?q=Natural+Language+Processing'],
             'GS-Unknown-Graphs':['http://scholar.google.com/scholar?q=Unknown+Graphs'],
             'GS-Physical-Graphs':['http://scholar.google.com/scholar?q=Physical+Graphs'],
             'GS-Model-Based-Diagnosis':['http://scholar.google.com/scholar?q=Model+Based+Diagnosis'],
             'GS-Man-Machine-Interface':['http://scholar.google.com/scholar?q=Man+Machine+Interface'],
              'GS-Information+Systems':['http://scholar.google.com/scholar?q=Information+Systems'],
             'GS-Zero-Knowledge':['http://scholar.google.com/scholar?q=Zero+Knowledge'],
             'GS-Heuristic-Internet':['http://scholar.google.com/scholar?q=Internet'],
             'GS-Heuristic-Encryption':['http://scholar.google.com/scholar?q=Encryption'],
             'GS-Heuristic-Search-Clique':['http://scholar.google.com/scholar?q=Heuristic+Search'],
             'GS-Heuristic-Data-Mining':['http://scholar.google.com/scholar?q=Data+Mining'],
             'GS-Sublinear-Algorithms':['http://scholar.google.com/scholar?q=Sublinear+Algorithms'],
             'GS-Software-Engineering':['http://scholar.google.com/scholar?q=Software+Engineering'],
             'GS-Data-Fusion':['http://scholar.google.com/scholar?q=Data+Fusion'],
             'GS-MDP':['http://scholar.google.com/scholar?q=MDP'],
             'GS-Search-Clique':['http://scholar.google.com/scholar?q=Search+Clique'],
             'GS-Lookahead':['http://scholar.google.com/scholar?q=Lookahead']}

#    seeds = {'citeseer-Search-Clique':['http://citeseer.ist.psu.edu/cis?q=Search+Clique&cs=1'],
#             'citeseer-Arificial-Intelligence-Clique':['http://citeseer.ist.psu.edu/cis?q=Arificial+Intelligence+Clique&cs=1'],
#             'citeseer-Heuristic-Search-Clique':['http://citeseer.ist.psu.edu/cis?q=Heuristic+Search&cs=1'],
#             'citeseer-Heuristic-MultiAgent':['http://citeseer.ist.psu.edu/cis?q=MultiAgent&cs=1'],
#             'citeseer-Heuristic-Israel':['http://citeseer.ist.psu.edu/cis?q=Israel&cs=1'],
#             'citeseer-Heuristic-Online-Algorithms':['http://citeseer.ist.psu.edu/cis?q=Online+Algorithms&cs=1'],
#             'citeseer-Heuristic-Robot-Navigation':['http://citeseer.ist.psu.edu/cis?q=Robot+Navigation&cs=1'],
#             'citeseer-Heuristic-Subgraph-Isomorphism':['http://citeseer.ist.psu.edu/cis?q=Subgraph+Isomorphism&cs=1'],
#             'citeseer-Heuristic-webgraph':['http://citeseer.ist.psu.edu/cis?q=webgraph&cs=1'],
#             'citeseer-Heuristic-Internet':['http://citeseer.ist.psu.edu/cis?q=Internet&cs=1'],
#             'citeseer-Heuristic-Social-Graph':['http://citeseer.ist.psu.edu/cis?q=Social+Graph&cs=1'],
#             'citeseer-Heuristic-Natural-Language-Processing':['http://citeseer.ist.psu.edu/cis?q=Natural+Language+Processing&cs=1'],
#             'citeseer-Heuristic-Unknown-Graphs':['http://citeseer.ist.psu.edu/cis?q=Unknown+Graphs&cs=1'],
#             'citeseer-Heuristic-Physical-Graphs':['http://citeseer.ist.psu.edu/cis?q=Physical+Graphs&cs=1'],
#             'citeseer-Heuristic-Model-Based-Diagnosis':['http://citeseer.ist.psu.edu/cis?q=Model+Based+Diagnosis&cs=1'],
#             'citeseer-Heuristic-Man-Machine-Interface':['http://citeseer.ist.psu.edu/cis?q=Man+Machine+Interface&cs=1'],
#             'citeseer-Heuristic-Information+Systems':['http://citeseer.ist.psu.edu/cis?q=Information+Systems&cs=1'],
#             'citeseer-Heuristic-Zero-Knowledge':['http://citeseer.ist.psu.edu/cis?q=Zero+Knowledge&cs=1'],        
#             'citeseer-Heuristic-Encryption':['http://citeseer.ist.psu.edu/cis?q=Encryption&cs=1'],
#             'citeseer-Heuristic-Sublinear-Algorithms':['http://citeseer.ist.psu.edu/cis?q=Sublinear+Algorithms&cs=1'],
#             'citeseer-Heuristic-Software-Engineering':['http://citeseer.ist.psu.edu/cis?q=Software+Engineering&cs=1'],
#             'citeseer-Heuristic-Data-Mining':['http://citeseer.ist.psu.edu/cis?q=Data+Mining&cs=1'],
#             'citeseer-Heuristic-Data-Fusion':['http://citeseer.ist.psu.edu/cis?q=Data+Fusion&cs=1'],
#             'citeseer-Heuristic-MDP':['http://citeseer.ist.psu.edu/cis?q=MDP&cs=1'],
#             'citeseer-Lookahead':['http://citeseer.ist.psu.edu/cis?q=Lookahead&cs=1']}
            
    fetcherThreads = 1
    maxUrlsToCrawl = 300
    timeOut = 5
    delay = 1        
    timerManager.timer_manager.create_timer("processSite")
    file_format = 'dot'
    keys = list(seeds.keys())
    for seed_index in xrange(len(keys)):
        seed = keys[seed_index]
        out_file_name = "%s/%s-%s" % (RESOURCE_WEBGRAPH_DIR,seed,maxUrlsToCrawl)   
        seed = {seed:seeds[seed]}        
        crawler = orchid.Orchid(seed, fetcherThreads, maxUrlsToCrawl, timeOut, delay,\
#                            analyzer = webgraphs.wikiAnalyzer.wikiAnalyzer,\
                            analyzer = webgraphs.googleScholarAnalyzer.googleScholarAnalyzer,\
#                            analyzer = webgraphs.citeseerAnalyzer.citeseerAnalyzer,\
#                            analyzer = webgraphs.webgraphCrawler.webgraphCrawler,                            
                            args = [out_file_name , file_format])
        orchid.Orchid.ignore_robots = True    
        crawler.crawl()
        
        # Added random sleep time to avoid tobot detection
        sleep_time = (random.Random().random()*30)
        time.sleep(sleep_time)
    
    for timing in timerManager.timer_manager.get_times("processSite"):
        print timing
    print "Done!"
    

if __name__ == '__main__':
    main()